{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6f6e72e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词典中的最大词长： 5\n",
      "正向最大匹配法FMM，分词结果： ['南京市长', '江', '大桥', '非常宏伟']\n",
      "逆向最大匹配法RMM，分词结果： ['南京市', '长江大桥', '非常宏伟']\n",
      "双向最大匹配法BMM，分词结果： ['南京市', '长江大桥', '非常宏伟']\n"
     ]
    }
   ],
   "source": [
    "import string, re\n",
    "def load_dict(filename):\t\t\t#读取字典并计算最大词长           def 函数  调用函数 函数名\n",
    "\tf = open(filename, 'r', encoding = 'utf8').read()\n",
    "\tmaxLen = 1\n",
    "\tdict = f.split(\"\\n\")\t\t\t#将词典按换行符进行分割\n",
    "\tfor i in dict:\t\t\t\t\t#计算最大词长\n",
    "\t\tif len(i)>maxLen:\n",
    "\t\t\tmaxLen = len(i)\n",
    "\treturn dict, maxLen;\n",
    "def FMM(dict, maxLen, text):\t\t\t#定义正向最大匹配函数\n",
    "\tcut_word = []\n",
    "\tstart = 0\t\t\t\t#变量start用于记录匹配字符串的起始位置\n",
    "\ttextLen = len(text)\n",
    "\twhile start != textLen:\n",
    "\t\t#变量index用于记录匹配字符串的结束位置\n",
    "\t\tindex = start + maxLen\n",
    "\t\tif index > len(text): \t\t#如果index大于待分词文本长度时\n",
    "\t\t\tindex = len(text)\n",
    "\t\tfor i in range(maxLen):\n",
    "\t\t\tif(text[start:index] in dict)or(len(text[start:\n",
    "index]) == 1):\t\t\t\t#如果匹配字符串在词典中或匹配字符串的长度为1\n",
    "\t\t\t\tcut_word.append(text[start:index])\n",
    "\t\t\t\tstart = index  5\n",
    "\t\t\t\tbreak\n",
    "\t\t\tindex -= 1   #index=index-1\n",
    "\treturn cut_word\n",
    "def RMM(dict, maxLen, text): \t\t\t#定义逆向最大匹配函数\n",
    "\tcut_word = []\n",
    "\ttextLen = len(text)\n",
    "\twhile textLen > 0:\n",
    "\t\tj = 0\n",
    "\t\tfor size in range(maxLen, 0, -1):\t#从最大词长开始，每次减1\n",
    "\t\t\t#如果待分词文本长度小于匹配字符串的长度，则结束本次循环\n",
    "\t\t\tif textLen < size:\n",
    "\t\t\t\tcontinue\n",
    "\t\t\tcutword = text[textLen-size:textLen]\t\t#切分字符串\n",
    "\t\t\tif cutword in dict:\n",
    "\t\t\t\tcut_word.append(cutword)\n",
    "\t\t\t\ttextLen -= size\t\t\t\t\t\t\t#更新文本长度\n",
    "\t\t\t\tj += 1\n",
    "\t\t\t\tbreak\n",
    "\t\tif j == 0:\n",
    "\t\t\ttextLen -= 1\n",
    "\tcut_word = list(reversed(cut_word))\t\t\t\t#反转列表\n",
    "\treturn cut_word\n",
    "def BMM(dict, maxLen, text): \t\t\t#定义双向最大匹配函数\n",
    "\tfmm = FMM(dict, maxLen, text)\t\t#调用正向最大匹配函数\n",
    "\trmm = RMM(dict, maxLen, text)\t\t#调用逆向最大匹配函数\n",
    "\t#如果分词数量不相等，选择分词数量较少的一组\n",
    "\tif len(fmm) != len(rmm): \n",
    "\t\tif len(fmm) < len(rmm):\n",
    "\t\t\treturn fmm\n",
    "\t\telse:\n",
    "\t\t\treturn rmm\n",
    "\telse:\t\t\t\t\t\t\t#如果分词数量相等\n",
    "\t\tif fmm == rmm:\t\t\t#如果分词结果完全一致，随机选择一组\n",
    "\t\t\treturn fmm\n",
    "\t\telse:\t\t\t\t#如果分词结果不一致，选择单字数量较少的一组\n",
    "\t\t\tfmm_1num = len ([i for i in fmm if len(i) == 1])\n",
    "\t\t\trmm_1num = len ([i for i in rmm if len(i) == 1])\n",
    "\t\t\treturn fmm_1num if fmm_1num < rmm_1num else rmm_1num\n",
    "def main():\t\t\t\t#主函数\n",
    "\tdict,maxLen = load_dict('data/dict1.txt')#路径\n",
    "\tprint(\"词典中的最大词长：\", maxLen)\n",
    "\ttext = \"南京市长江大桥非常宏伟\"\n",
    "\tfmm_cut = FMM(dict, maxLen, text)\n",
    "\tprint(\"正向最大匹配法FMM，分词结果：\", fmm_cut)\n",
    "\trmm_cut = RMM(dict, maxLen, text)\n",
    "\tprint(\"逆向最大匹配法RMM，分词结果：\", rmm_cut)\n",
    "\tbmm_cut = BMM(dict, maxLen, text)\n",
    "\tprint(\"双向最大匹配法BMM，分词结果：\", bmm_cut)\n",
    "if __name__ == '__main__':  程序入口\n",
    "\tmain()     #调用函数   函数名()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83827970",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f16fb86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2666fe36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词典中的最大词长： 5\n",
      "正向最大匹配法FMM，分词结果： ['南京市长', '江', '大桥', '非常宏伟']\n",
      "逆向最大匹配法RMM，分词结果： ['南京市', '长江大桥', '非常宏伟']\n",
      "双向最大匹配法BMM，分词结果： ['南京市', '长江大桥', '非常宏伟']\n"
     ]
    }
   ],
   "source": [
    "import string, re\n",
    "def load_dict(filename):\t\t\t#读取字典并计算最大词长\n",
    "\tf = open(filename, 'r', encoding = 'utf8').read()\n",
    "\tmaxLen = 1\n",
    "\tdict = f.split(\"\\n\")\t\t\t#将词典按换行符进行分割\n",
    "\tfor i in dict:\t\t\t\t\t#计算最大词长\n",
    "\t\tif len(i)>maxLen:\n",
    "\t\t\tmaxLen = len(i)\n",
    "\treturn dict, maxLen;\n",
    "def FMM(dict, maxLen, text):\t\t\t#定义正向最大匹配函数\n",
    "\tcut_word = []\n",
    "\tstart = 0\t\t\t\t#变量start用于记录匹配字符串的起始位置\n",
    "\ttextLen = len(text)\n",
    "\twhile start != textLen:\n",
    "\t\t#变量index用于记录匹配字符串的结束位置\n",
    "\t\tindex = start + maxLen\n",
    "\t\tif index > len(text): \t\t#如果index大于待分词文本长度时\n",
    "\t\t\tindex = len(text)\n",
    "\t\tfor i in range(maxLen):\n",
    "\t\t\tif(text[start:index] in dict)or(len(text[start:\n",
    "index]) == 1):\t\t\t\t#如果匹配字符串在词典中或匹配字符串的长度为1\n",
    "\t\t\t\tcut_word.append(text[start:index])\n",
    "\t\t\t\tstart = index\n",
    "\t\t\t\tbreak\n",
    "\t\t\tindex -= 1\n",
    "\treturn cut_word\n",
    "def RMM(dict, maxLen, text): \t\t\t#定义逆向最大匹配函数\n",
    "\tcut_word = []\n",
    "\ttextLen = len(text)\n",
    "\twhile textLen > 0:\n",
    "\t\tj = 0\n",
    "\t\tfor size in range(maxLen, 0, -1):\t#从最大词长开始，每次减1\n",
    "\t\t\t#如果待分词文本长度小于匹配字符串的长度，则结束本次循环\n",
    "\t\t\tif textLen < size:\n",
    "\t\t\t\tcontinue\n",
    "\t\t\tcutword = text[textLen-size:textLen]\t\t#切分字符串\n",
    "\t\t\tif cutword in dict:\n",
    "\t\t\t\tcut_word.append(cutword)\n",
    "\t\t\t\ttextLen -= size\t\t\t\t\t\t\t#更新文本长度\n",
    "\t\t\t\tj += 1\n",
    "\t\t\t\tbreak\n",
    "\t\tif j == 0:\n",
    "\t\t\ttextLen -= 1\n",
    "\tcut_word = list(reversed(cut_word))\t\t\t\t#反转列表\n",
    "\treturn cut_word\n",
    "def BMM(dict, maxLen, text): \t\t\t#定义双向最大匹配函数\n",
    "\tfmm = FMM(dict, maxLen, text)\t\t#调用正向最大匹配函数\n",
    "\trmm = RMM(dict, maxLen, text)\t\t#调用逆向最大匹配函数\n",
    "\t#如果分词数量不相等，选择分词数量较少的一组\n",
    "\tif len(fmm) != len(rmm): \n",
    "\t\tif len(fmm) < len(rmm):\n",
    "\t\t\treturn fmm\n",
    "\t\telse:\n",
    "\t\t\treturn rmm\n",
    "\telse:\t\t\t\t\t\t\t#如果分词数量相等\n",
    "\t\tif fmm == rmm:\t\t\t#如果分词结果完全一致，随机选择一组\n",
    "\t\t\treturn fmm\n",
    "\t\telse:\t\t\t\t#如果分词结果不一致，选择单字数量较少的一组\n",
    "\t\t\tfmm_1num = len ([i for i in fmm if len(i) == 1])\n",
    "\t\t\trmm_1num = len ([i for i in rmm if len(i) == 1])\n",
    "\t\t\treturn fmm_1num if fmm_1num < rmm_1num else rmm_1num\n",
    "def main():\t\t\t\t#主函数\n",
    "\tdict,maxLen = load_dict('data/dict1.txt')\n",
    "\tprint(\"词典中的最大词长：\", maxLen)\n",
    "\ttext = \"南京市长江大桥非常宏伟\"\n",
    "\tfmm_cut = FMM(dict, maxLen, text)\n",
    "\tprint(\"正向最大匹配法FMM，分词结果：\", fmm_cut)\n",
    "\trmm_cut = RMM(dict, maxLen, text)\n",
    "\tprint(\"逆向最大匹配法RMM，分词结果：\", rmm_cut)\n",
    "\tbmm_cut = BMM(dict, maxLen, text)\n",
    "\tprint(\"双向最大匹配法BMM，分词结果：\", bmm_cut)\n",
    "if __name__ == '__main__':\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "92b96d5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词典中的最大词长： 5\n",
      "正向最大匹配法FMM，分词结果： ['南京市长']\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'str' object has no attribute 'append'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-1b760c8dd93c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     80\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"双向最大匹配法BMM，分词结果：\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbmm_cut\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'__main__'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 82\u001b[1;33m         \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-10-1b760c8dd93c>\u001b[0m in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m     75\u001b[0m         \u001b[0mfmm_cut\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFMM\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxLen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"正向最大匹配法FMM，分词结果：\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfmm_cut\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 77\u001b[1;33m         \u001b[0mrmm_cut\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mRMM\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxLen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     78\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"逆向最大匹配法RMM，分词结果：\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrmm_cut\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m         \u001b[0mbmm_cut\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBMM\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxLen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-10-1b760c8dd93c>\u001b[0m in \u001b[0;36mRMM\u001b[1;34m(dict, maxLen, text)\u001b[0m\n\u001b[0;32m     33\u001b[0m             \u001b[0mcut_word\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtextLen\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mtextLen\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mcut_word\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 35\u001b[1;33m                 \u001b[0mcut_word\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcut_word\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     36\u001b[0m                 \u001b[0mtextLen\u001b[0m \u001b[1;33m-=\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m                 \u001b[0mj\u001b[0m\u001b[1;33m+=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'str' object has no attribute 'append'"
     ]
    }
   ],
   "source": [
    "import string, re\n",
    "def load_dict(filename):\n",
    "    f =open(filename, 'r', encoding = 'utf-8').read()\n",
    "    maxLen =1\n",
    "    dict = f.split(\"\\n\")\n",
    "    for i  in dict:\n",
    "        if len(i)>maxLen:\n",
    "            maxLen = len(i)\n",
    "    return dict, maxLen;\n",
    "def FMM(dict, maxLen, text):\n",
    "    cut_word =[]\n",
    "    start = 0\n",
    "    textLen = len(text)\n",
    "    while start !=textLen:\n",
    "        index = start+ maxLen\n",
    "        if index >len(text):\n",
    "            index = len(text)\n",
    "        for i in range(maxLen):\n",
    "            if(text[start:index] in dict)or(len(text[start : index]) ==1):\n",
    "                cut_word.append(text[start:index])\n",
    "                start = index\n",
    "                break\n",
    "            index -=1\n",
    "        return cut_word\n",
    "def RMM(dict, maxLen, text):\n",
    "    cut_word =[]\n",
    "    textLen = len(text)\n",
    "    while textLen >0:\n",
    "        j = 0\n",
    "        for size in range(maxLen, 0, -1):\n",
    "            if textLen < size:\n",
    "                continue\n",
    "            cut_word = text[textLen-size:textLen]\n",
    "            if cut_word in dict:\n",
    "                cut_word.append(cut_word)\n",
    "                textLen -= size\n",
    "                j+=1\n",
    "                break\n",
    "        if j ==0:\n",
    "            textLen -=1\n",
    "            cut_word = list(reversed(cut_word))\n",
    "            return cut_word\n",
    "def BMM(dict, maxLen, text):\n",
    "    fmm = FMM(dict, maxLen,text)\n",
    "    rmm = RMM(dict, maxLen, text)\n",
    "    if len(fmm) !=len(rmm):\n",
    "        if len(fmm)<len(rmm):\n",
    "            return fmm\n",
    "        else:\n",
    "            return rmm\n",
    "    else:\n",
    "        if fmm == rmm:\n",
    "            return fmm\n",
    "        else:\n",
    "            fmm_lnum = len ([i for i in fmm if len(i) == 1])\n",
    "            rmm_lnum = len ([i for i in fmm if len(i) == 1])\n",
    "            return fmm_lnum if fmm_lnum < rmm_lnum else rmm_lnum\n",
    "# def main():\n",
    "#     dict,maxLen = load_dict('dict1.txt')\n",
    "#     print(\"词典中的最大词长:\",maxLen)\n",
    "#     text = \"南京市长江大桥非常宏伟\"\n",
    "#     fmm_cut= FMM(dict, maxLen, text)\n",
    "#     print(\"正向最大匹配法FMM,分词结果:\",fmm_cut)\n",
    "#     rmm_cut= RMM(dict, maxLen, text)\n",
    "#     print(\"逆向最大匹配法RMM,分词结果:\",rmm_cut)\n",
    "#     bmm_cut= BMM(dict, maxLen, text)\n",
    "#     print(\"双向最大匹配法BMM,分词结果:\",bmm_cut)\n",
    "# #if __name__ == '__main__':\n",
    "# if __name__ == '__main__':\n",
    "#     main()\n",
    "def main():\t\t\t\t#主函数\n",
    "\tdict,maxLen = load_dict('data/dict1.txt')\n",
    "\tprint(\"词典中的最大词长：\", maxLen)\n",
    "\ttext = \"南京市长江大桥非常宏伟\"\n",
    "\tfmm_cut = FMM(dict, maxLen, text)\n",
    "\tprint(\"正向最大匹配法FMM，分词结果：\", fmm_cut)\n",
    "\trmm_cut = RMM(dict, maxLen, text)\n",
    "\tprint(\"逆向最大匹配法RMM，分词结果：\", rmm_cut)\n",
    "\tbmm_cut = BMM(dict, maxLen, text)\n",
    "\tprint(\"双向最大匹配法BMM，分词结果：\", bmm_cut)\n",
    "if __name__ == '__main__':\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29ff2350",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
